import crawlertool as tool
from bs4 import BeautifulSoup
import time
import random
# pip install fake_useragent
from fake_useragent import UserAgent

'''
起点中文网小说收藏榜数据爬虫(玄幻小说排行榜)
'''
class SpiderQidianBookTypeList(tool.abc.SingleSpider):
    _HEADERS = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Host": "www.qidian.com",
        "Pragma": "no-cache",
        "Referer": "https://www.qidian.com/rank/collect?style=2&chn=12&page=1",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "same-origin",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": UserAgent().chrome
    }

    def running(self, page_num):
        book_list = []
        for page_size in range(1, page_num):
            url = "https://www.qidian.com/rank/collect/chn21/page{0}/".format(page_size)
            response = tool.do_request(url, headers=self._HEADERS)
            # 解析答案
            bs = BeautifulSoup(response.content.decode(), "lxml")
            for book_label in bs.select("#rank-view-list > div > ul > li"):
                book_list.append({
                    # 小说排名
                    "book_rank": book_label.select_one("li > .book-img-box > span").text,
                    # 小说封面
                    "book_cover": 'https:' + book_label.select_one("li > .book-img-box > a > img")['src'],
                    # 小说名称
                    "book_name": book_label.select_one("li > .book-mid-info > h2 > a").text,
                    # 作者姓名
                    "author_name": book_label.select_one("li > .book-mid-info > .author > .name").text,
                    # 作品类型
                    "book_type": book_label.select("li > .book-mid-info > .author > a")[1].text + '-' + book_label.select("li > .book-mid-info > .author > a")[2].text,
                    # 小说状态
                    "book_status": book_label.select_one("li > .book-mid-info > .author > span").text,
                    # 小说简介
                    "book_intro": book_label.select_one("li > .book-mid-info > .intro").text,
                    # 最新章节
                    "book_latest_chapter": book_label.select_one("li > .book-mid-info > .update > a").text,
                    # 更新时间
                    "update_time": book_label.select_one("li > .book-mid-info > .update > span").text,
                })
            time.sleep(random.randint(3, 5))
        return book_list


# ------------------- 单元测试 -------------------
if __name__ == "__main__":
    print(SpiderQidianBookTypeList().running(3))
